{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copyright (c) 2020, NVIDIA CORPORATION.\n",
    "Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "you may not use this file except in compliance with the License.\n",
    "You may obtain a copy of the License at\n",
    "    http://www.apache.org/licenses/LICENSE-2.0\n",
    "Unless required by applicable law or agreed to in writing, software\n",
    "distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "See the License for the specific language governing permissions and\n",
    "limitations under the License."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, time\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "start = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd, numpy as np, gc\n",
    "from datetime import datetime\n",
    "import joblib\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.max_rows', 500)\n",
    "import cudf, cupy, time\n",
    "cudf.__version__\n",
    "\n",
    "startNB = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numba import jit, njit, prange\n",
    "from sklearn.metrics import precision_recall_curve, auc, log_loss\n",
    "\n",
    "def compute_prauc(gt, pred, nafill=True):\n",
    "    if nafill:\n",
    "        pred[ np.isnan(pred) ] = np.nanmean( pred )\n",
    "    prec, recall, thresh = precision_recall_curve(gt, pred)\n",
    "    prauc = auc(recall, prec)\n",
    "    return prauc\n",
    "\n",
    "@jit\n",
    "def fast_auc(y_true, y_prob):\n",
    "    y_true = np.asarray(y_true)\n",
    "    y_true = y_true[np.argsort(y_prob)]\n",
    "    nfalse = 0\n",
    "    auc = 0\n",
    "    n = len(y_true)\n",
    "    for i in range(n):\n",
    "        y_i = y_true[i]\n",
    "        nfalse += (1 - y_i)\n",
    "        auc += y_i * nfalse\n",
    "    auc /= (nfalse * (n - nfalse))\n",
    "    return auc\n",
    "\n",
    "@njit\n",
    "def numba_log_loss(y,x):\n",
    "    n = x.shape[0]\n",
    "    ll = 0.\n",
    "    for i in prange(n):\n",
    "        if y[i]<=0.:\n",
    "            ll += np.log(1-x[i] + 1e-15 )\n",
    "        else:\n",
    "            ll += np.log(x[i] + 1e-15)\n",
    "    return -ll / n\n",
    "\n",
    "def compute_rce(gt , pred, nafill=True, verbose=0):\n",
    "    if nafill:\n",
    "        pred[ np.isnan(pred) ] = np.nanmean( pred )\n",
    "        \n",
    "    cross_entropy = numba_log_loss( gt, pred  )\n",
    "    \n",
    "    yt = np.mean(gt>0)     \n",
    "    strawman_cross_entropy = -(yt*np.log(yt) + (1 - yt)*np.log(1 - yt))\n",
    "    \n",
    "    if verbose:\n",
    "        print( \"logloss: {0:.5f} / {1:.5f} = {2:.5f}\".format(cross_entropy, strawman_cross_entropy, cross_entropy/strawman_cross_entropy))\n",
    "        print( 'mean:    {0:.5f} / {1:.5f}'.format( np.nanmean( pred ) , yt  ) )\n",
    "    \n",
    "    return (1.0 - cross_entropy/strawman_cross_entropy)*100.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_memory( df ):\n",
    "    features = df.columns\n",
    "    for i in range( df.shape[1] ):\n",
    "        if df.dtypes[i] == 'uint8':\n",
    "            df[features[i]] = df[features[i]].astype( np.int8 )\n",
    "            gc.collect()\n",
    "        elif df.dtypes[i] == 'bool':\n",
    "            df[features[i]] = df[features[i]].astype( np.int8 )\n",
    "            gc.collect()\n",
    "        elif df.dtypes[i] == 'uint32':\n",
    "            df[features[i]] = df[features[i]].astype( np.int32 )\n",
    "            gc.collect()\n",
    "        elif df.dtypes[i] == 'int64':\n",
    "            df[features[i]] = df[features[i]].astype( np.int32 )\n",
    "            gc.collect()\n",
    "        elif df.dtypes[i] == 'float64':\n",
    "            df[features[i]] = df[features[i]].astype( np.float32 )\n",
    "            gc.collect()\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 53.7 s, sys: 32.9 s, total: 1min 26s\n",
      "Wall time: 5.84 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((121386431, 25), (12434735, 25), (12434838, 25))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train = pd.read_parquet( '../preprocessings/train-1.parquet' )\n",
    "test0 = pd.read_parquet( '../preprocessings/test-0.parquet' )\n",
    "test1 = pd.read_parquet( '../preprocessings/test-1.parquet' )\n",
    "gc.collect()\n",
    "\n",
    "train['tr'] = 0\n",
    "test0['tr'] = 1\n",
    "test1['tr'] = 2\n",
    "\n",
    "train.shape, test0.shape, test1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(146256004, 25)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.concat( (train,test0,test1), sort=False )\n",
    "gc.collect()\n",
    "del test0, test1\n",
    "gc.collect()\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 20.1 s, sys: 12.1 s, total: 32.1 s\n",
      "Wall time: 32.1 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train = train.sort_values('id').reset_index(drop=True) \n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[ train.reply>0, 'reply' ] = 1\n",
    "train.loc[ train.retweet>0, 'retweet' ] = 1\n",
    "train.loc[ train.retweet_comment>0, 'retweet_comment' ] = 1\n",
    "train.loc[ train.like>0, 'like' ] = 1\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hashtags</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>links</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>a_account_creation</th>\n",
       "      <th>b_user_id</th>\n",
       "      <th>b_follower_count</th>\n",
       "      <th>b_following_count</th>\n",
       "      <th>b_is_verified</th>\n",
       "      <th>b_account_creation</th>\n",
       "      <th>b_follows_a</th>\n",
       "      <th>reply</th>\n",
       "      <th>retweet</th>\n",
       "      <th>retweet_comment</th>\n",
       "      <th>like</th>\n",
       "      <th>id</th>\n",
       "      <th>tr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1581262691</td>\n",
       "      <td>0</td>\n",
       "      <td>986</td>\n",
       "      <td>1201</td>\n",
       "      <td>False</td>\n",
       "      <td>1274269909</td>\n",
       "      <td>6237935</td>\n",
       "      <td>94</td>\n",
       "      <td>648</td>\n",
       "      <td>False</td>\n",
       "      <td>1478011810</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>1581497241</td>\n",
       "      <td>1</td>\n",
       "      <td>1225</td>\n",
       "      <td>677</td>\n",
       "      <td>False</td>\n",
       "      <td>1255778244</td>\n",
       "      <td>879586</td>\n",
       "      <td>1139</td>\n",
       "      <td>46</td>\n",
       "      <td>False</td>\n",
       "      <td>1540395738</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1580978528</td>\n",
       "      <td>2</td>\n",
       "      <td>3016</td>\n",
       "      <td>1623</td>\n",
       "      <td>False</td>\n",
       "      <td>1313450503</td>\n",
       "      <td>647103</td>\n",
       "      <td>780</td>\n",
       "      <td>440</td>\n",
       "      <td>False</td>\n",
       "      <td>1432084055</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>1581321849</td>\n",
       "      <td>3</td>\n",
       "      <td>2121</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>1547717153</td>\n",
       "      <td>13774339</td>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>1534313747</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1580956787</td>\n",
       "      <td>4</td>\n",
       "      <td>813505</td>\n",
       "      <td>200</td>\n",
       "      <td>True</td>\n",
       "      <td>1476348838</td>\n",
       "      <td>13774340</td>\n",
       "      <td>171</td>\n",
       "      <td>388</td>\n",
       "      <td>False</td>\n",
       "      <td>1490166885</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   hashtags  tweet_id  media  links  domains  tweet_type  language  \\\n",
       "0         0         0      0      0        0           2        11   \n",
       "1         3         1      0      0        0           1        11   \n",
       "2         0         2      0      1        1           2        11   \n",
       "3         0         3      0      0        0           1        54   \n",
       "4         0         4      5      0        0           2        11   \n",
       "\n",
       "    timestamp  a_user_id  a_follower_count  a_following_count  a_is_verified  \\\n",
       "0  1581262691          0               986               1201          False   \n",
       "1  1581497241          1              1225                677          False   \n",
       "2  1580978528          2              3016               1623          False   \n",
       "3  1581321849          3              2121                 16          False   \n",
       "4  1580956787          4            813505                200           True   \n",
       "\n",
       "   a_account_creation  b_user_id  b_follower_count  b_following_count  \\\n",
       "0          1274269909    6237935                94                648   \n",
       "1          1255778244     879586              1139                 46   \n",
       "2          1313450503     647103               780                440   \n",
       "3          1547717153   13774339                 1                 45   \n",
       "4          1476348838   13774340               171                388   \n",
       "\n",
       "   b_is_verified  b_account_creation  b_follows_a  reply  retweet  \\\n",
       "0          False          1478011810        False      0        0   \n",
       "1          False          1540395738         True      0        1   \n",
       "2          False          1432084055         True      0        0   \n",
       "3          False          1534313747        False      0        0   \n",
       "4          False          1490166885        False      0        0   \n",
       "\n",
       "   retweet_comment  like  id  tr  \n",
       "0                0     0   0   0  \n",
       "1                0     1   1   0  \n",
       "2                0     1   2   0  \n",
       "3                0     1   3   0  \n",
       "4                0     1   4   0  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "save_memory(train)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "hashtags              int32\n",
       "tweet_id              int32\n",
       "media                  int8\n",
       "links                 int32\n",
       "domains               int32\n",
       "tweet_type             int8\n",
       "language               int8\n",
       "timestamp             int32\n",
       "a_user_id             int32\n",
       "a_follower_count      int32\n",
       "a_following_count     int32\n",
       "a_is_verified          int8\n",
       "a_account_creation    int32\n",
       "b_user_id             int32\n",
       "b_follower_count      int32\n",
       "b_following_count     int32\n",
       "b_is_verified          int8\n",
       "b_account_creation    int32\n",
       "b_follows_a            int8\n",
       "reply                 int32\n",
       "retweet               int32\n",
       "retweet_comment       int32\n",
       "like                  int32\n",
       "id                    int32\n",
       "tr                    int32\n",
       "dtype: object"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dt_day\n",
       "6     16645598\n",
       "7     17604774\n",
       "8     18291076\n",
       "9     17594952\n",
       "10    17778468\n",
       "11    17395299\n",
       "12    16076264\n",
       "13     3566976\n",
       "14     3443360\n",
       "15     3676153\n",
       "16     3575402\n",
       "17     3827601\n",
       "18     3555061\n",
       "19     3225020\n",
       "Name: id, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['dt_day']  = pd.to_datetime( train['timestamp'] , unit='s' ).dt.day.values.astype( np.int8 )\n",
    "train['dt_dow']  = pd.to_datetime( train['timestamp'] , unit='s' ).dt.dayofweek.values.astype( np.int8 )\n",
    "train['dt_hour'] = pd.to_datetime( train['timestamp'] , unit='s' ).dt.hour.values.astype( np.int8 )\n",
    "_=gc.collect()\n",
    "train.groupby('dt_day')['id'].agg('count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>a_count_combined</th>\n",
       "      <th>a_user_fer_count_delta_time</th>\n",
       "      <th>a_user_fing_count_delta_time</th>\n",
       "      <th>a_user_fering_count_delta_time</th>\n",
       "      <th>a_user_fing_count_mode</th>\n",
       "      <th>a_user_fer_count_mode</th>\n",
       "      <th>a_user_fering_count_mode</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  a_count_combined  a_user_fer_count_delta_time  \\\n",
       "0   0                18                           -1   \n",
       "1   1                 2                            1   \n",
       "2   2                 2                            1   \n",
       "3   3                 2                            1   \n",
       "4   4                 2                            1   \n",
       "\n",
       "   a_user_fing_count_delta_time  a_user_fering_count_delta_time  \\\n",
       "0                             0                              -1   \n",
       "1                             1                               1   \n",
       "2                             1                               1   \n",
       "3                             1                               1   \n",
       "4                             1                               1   \n",
       "\n",
       "   a_user_fing_count_mode  a_user_fer_count_mode  a_user_fering_count_mode  \n",
       "0                       0                      0                         0  \n",
       "1                       1                      1                         1  \n",
       "2                       1                      1                         1  \n",
       "3                       1                      1                         1  \n",
       "4                       1                      1                         1  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt = pd.read_parquet( '../preprocessings/a_count_combined-final.parquet' )\n",
    "dt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hashtags</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>links</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>a_account_creation</th>\n",
       "      <th>b_user_id</th>\n",
       "      <th>b_follower_count</th>\n",
       "      <th>b_following_count</th>\n",
       "      <th>b_is_verified</th>\n",
       "      <th>b_account_creation</th>\n",
       "      <th>b_follows_a</th>\n",
       "      <th>reply</th>\n",
       "      <th>retweet</th>\n",
       "      <th>retweet_comment</th>\n",
       "      <th>like</th>\n",
       "      <th>id</th>\n",
       "      <th>tr</th>\n",
       "      <th>dt_day</th>\n",
       "      <th>dt_dow</th>\n",
       "      <th>dt_hour</th>\n",
       "      <th>a_count_combined</th>\n",
       "      <th>a_user_fer_count_delta_time</th>\n",
       "      <th>a_user_fing_count_delta_time</th>\n",
       "      <th>a_user_fering_count_delta_time</th>\n",
       "      <th>a_user_fing_count_mode</th>\n",
       "      <th>a_user_fer_count_mode</th>\n",
       "      <th>a_user_fering_count_mode</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1581262691</td>\n",
       "      <td>0</td>\n",
       "      <td>986</td>\n",
       "      <td>1201</td>\n",
       "      <td>0</td>\n",
       "      <td>1274269909</td>\n",
       "      <td>6237935</td>\n",
       "      <td>94</td>\n",
       "      <td>648</td>\n",
       "      <td>0</td>\n",
       "      <td>1478011810</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "      <td>18</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>1581497241</td>\n",
       "      <td>1</td>\n",
       "      <td>1225</td>\n",
       "      <td>677</td>\n",
       "      <td>0</td>\n",
       "      <td>1255778244</td>\n",
       "      <td>879586</td>\n",
       "      <td>1139</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>1540395738</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1580978528</td>\n",
       "      <td>2</td>\n",
       "      <td>3016</td>\n",
       "      <td>1623</td>\n",
       "      <td>0</td>\n",
       "      <td>1313450503</td>\n",
       "      <td>647103</td>\n",
       "      <td>780</td>\n",
       "      <td>440</td>\n",
       "      <td>0</td>\n",
       "      <td>1432084055</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>1581321849</td>\n",
       "      <td>3</td>\n",
       "      <td>2121</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>1547717153</td>\n",
       "      <td>13774339</td>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>0</td>\n",
       "      <td>1534313747</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1580956787</td>\n",
       "      <td>4</td>\n",
       "      <td>813505</td>\n",
       "      <td>200</td>\n",
       "      <td>1</td>\n",
       "      <td>1476348838</td>\n",
       "      <td>13774340</td>\n",
       "      <td>171</td>\n",
       "      <td>388</td>\n",
       "      <td>0</td>\n",
       "      <td>1490166885</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   hashtags  tweet_id  media  links  domains  tweet_type  language  \\\n",
       "0         0         0      0      0        0           2        11   \n",
       "1         3         1      0      0        0           1        11   \n",
       "2         0         2      0      1        1           2        11   \n",
       "3         0         3      0      0        0           1        54   \n",
       "4         0         4      5      0        0           2        11   \n",
       "\n",
       "    timestamp  a_user_id  a_follower_count  a_following_count  a_is_verified  \\\n",
       "0  1581262691          0               986               1201              0   \n",
       "1  1581497241          1              1225                677              0   \n",
       "2  1580978528          2              3016               1623              0   \n",
       "3  1581321849          3              2121                 16              0   \n",
       "4  1580956787          4            813505                200              1   \n",
       "\n",
       "   a_account_creation  b_user_id  b_follower_count  b_following_count  \\\n",
       "0          1274269909    6237935                94                648   \n",
       "1          1255778244     879586              1139                 46   \n",
       "2          1313450503     647103               780                440   \n",
       "3          1547717153   13774339                 1                 45   \n",
       "4          1476348838   13774340               171                388   \n",
       "\n",
       "   b_is_verified  b_account_creation  b_follows_a  reply  retweet  \\\n",
       "0              0          1478011810            0      0        0   \n",
       "1              0          1540395738            1      0        1   \n",
       "2              0          1432084055            1      0        0   \n",
       "3              0          1534313747            0      0        0   \n",
       "4              0          1490166885            0      0        0   \n",
       "\n",
       "   retweet_comment  like  id  tr  dt_day  dt_dow  dt_hour  a_count_combined  \\\n",
       "0                0     0   0   0       9       6       15                18   \n",
       "1                0     1   1   0      12       2        8                -9   \n",
       "2                0     1   2   0       6       3        8                 2   \n",
       "3                0     1   3   0      10       0        8                 2   \n",
       "4                0     1   4   0       6       3        2                 2   \n",
       "\n",
       "   a_user_fer_count_delta_time  a_user_fing_count_delta_time  \\\n",
       "0                           -1                             0   \n",
       "1                           -9                            -9   \n",
       "2                            1                             1   \n",
       "3                            1                             1   \n",
       "4                            1                             1   \n",
       "\n",
       "   a_user_fering_count_delta_time  a_user_fing_count_mode  \\\n",
       "0                              -1                       0   \n",
       "1                              -9                      -9   \n",
       "2                               1                       1   \n",
       "3                               1                       1   \n",
       "4                               1                       1   \n",
       "\n",
       "   a_user_fer_count_mode  a_user_fering_count_mode  \n",
       "0                      0                         0  \n",
       "1                     -9                        -9  \n",
       "2                      1                         1  \n",
       "3                      1                         1  \n",
       "4                      1                         1  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['a_count_combined']             = dt['a_count_combined']\n",
    "train['a_user_fer_count_delta_time']  = dt['a_user_fer_count_delta_time']\n",
    "train['a_user_fing_count_delta_time'] = dt['a_user_fing_count_delta_time']\n",
    "train['a_user_fering_count_delta_time']=dt['a_user_fering_count_delta_time']\n",
    "train['a_user_fing_count_mode']       = dt['a_user_fing_count_mode']\n",
    "train['a_user_fer_count_mode']        = dt['a_user_fer_count_mode']\n",
    "train['a_user_fering_count_mode']     = dt['a_user_fering_count_mode']\n",
    "\n",
    "train.loc[ (train.dt_day==12)|(train.dt_day==18) ,['a_count_combined','a_user_fer_count_delta_time','a_user_fing_count_delta_time','a_user_fering_count_delta_time','a_user_fing_count_mode','a_user_fer_count_mode','a_user_fering_count_mode']] = -9\n",
    "\n",
    "del dt; _=gc.collect()\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "hashtags                          int32\n",
       "tweet_id                          int32\n",
       "media                              int8\n",
       "links                             int32\n",
       "domains                           int32\n",
       "tweet_type                         int8\n",
       "language                           int8\n",
       "timestamp                         int32\n",
       "a_user_id                         int32\n",
       "a_follower_count                  int32\n",
       "a_following_count                 int32\n",
       "a_is_verified                      int8\n",
       "a_account_creation                int32\n",
       "b_user_id                         int32\n",
       "b_follower_count                  int32\n",
       "b_following_count                 int32\n",
       "b_is_verified                      int8\n",
       "b_account_creation                int32\n",
       "b_follows_a                        int8\n",
       "reply                             int32\n",
       "retweet                           int32\n",
       "retweet_comment                   int32\n",
       "like                              int32\n",
       "id                                int32\n",
       "tr                                int32\n",
       "dt_day                             int8\n",
       "dt_dow                             int8\n",
       "dt_hour                            int8\n",
       "a_count_combined                  int32\n",
       "a_user_fer_count_delta_time       int32\n",
       "a_user_fing_count_delta_time      int32\n",
       "a_user_fering_count_delta_time    int32\n",
       "a_user_fing_count_mode            int32\n",
       "a_user_fer_count_mode             int32\n",
       "a_user_fering_count_mode          int32\n",
       "dtype: object"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>count_ats</th>\n",
       "      <th>count_char</th>\n",
       "      <th>count_words</th>\n",
       "      <th>tw_hash</th>\n",
       "      <th>tw_freq_hash</th>\n",
       "      <th>tw_first_word</th>\n",
       "      <th>tw_second_word</th>\n",
       "      <th>tw_last_word</th>\n",
       "      <th>tw_llast_word</th>\n",
       "      <th>tw_len</th>\n",
       "      <th>tw_hash0</th>\n",
       "      <th>tw_hash1</th>\n",
       "      <th>tw_rt_uhash</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>68</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>182</td>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>661291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>105</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>103</td>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>198539</td>\n",
       "      <td>2048</td>\n",
       "      <td>616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>237</td>\n",
       "      <td>63</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  count_ats  count_char  count_words  tw_hash  tw_freq_hash  \\\n",
       "0   0          0          68           17        0             0   \n",
       "1   1          0         182           37        1             1   \n",
       "2   2          0         105           24        2             2   \n",
       "3   3          2         103           22        3             3   \n",
       "4   4          0         237           63        4             4   \n",
       "\n",
       "   tw_first_word  tw_second_word  tw_last_word  tw_llast_word  tw_len  \\\n",
       "0              0               0             0              0       7   \n",
       "1              1               1             1              1      12   \n",
       "2              2               2             2              2       6   \n",
       "3              3               3             3              3      10   \n",
       "4              4               4             4              4      16   \n",
       "\n",
       "   tw_hash0  tw_hash1  tw_rt_uhash  \n",
       "0         0         0            0  \n",
       "1         0         0       661291  \n",
       "2         0         0            0  \n",
       "3    198539      2048          616  \n",
       "4         0         0            0  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt = pd.read_parquet( '../preprocessings/text-processings-1.parquet' )\n",
    "dt.sort_values('id', inplace=True)\n",
    "dt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['count_ats']     = dt['count_ats']\n",
    "train['count_char']    = dt['count_char']\n",
    "train['count_words']   = dt['count_words']\n",
    "train['tw_hash']       = dt['tw_hash']\n",
    "train['tw_freq_hash']  = dt['tw_freq_hash']\n",
    "train['tw_first_word'] = dt['tw_first_word']\n",
    "train['tw_second_word']= dt['tw_second_word']\n",
    "train['tw_last_word']  = dt['tw_last_word']\n",
    "train['tw_llast_word'] = dt['tw_llast_word']\n",
    "train['tw_len']        = dt['tw_len']\n",
    "train['tw_hash0']      = dt['tw_hash0']\n",
    "train['tw_hash1']      = dt['tw_hash1']\n",
    "train['tw_rt_uhash']   = dt['tw_rt_uhash']\n",
    "del dt\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hashtags</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>links</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>a_account_creation</th>\n",
       "      <th>b_user_id</th>\n",
       "      <th>b_follower_count</th>\n",
       "      <th>b_following_count</th>\n",
       "      <th>b_is_verified</th>\n",
       "      <th>b_account_creation</th>\n",
       "      <th>b_follows_a</th>\n",
       "      <th>reply</th>\n",
       "      <th>retweet</th>\n",
       "      <th>retweet_comment</th>\n",
       "      <th>like</th>\n",
       "      <th>id</th>\n",
       "      <th>tr</th>\n",
       "      <th>dt_day</th>\n",
       "      <th>dt_dow</th>\n",
       "      <th>dt_hour</th>\n",
       "      <th>a_count_combined</th>\n",
       "      <th>a_user_fer_count_delta_time</th>\n",
       "      <th>a_user_fing_count_delta_time</th>\n",
       "      <th>a_user_fering_count_delta_time</th>\n",
       "      <th>a_user_fing_count_mode</th>\n",
       "      <th>a_user_fer_count_mode</th>\n",
       "      <th>a_user_fering_count_mode</th>\n",
       "      <th>count_ats</th>\n",
       "      <th>count_char</th>\n",
       "      <th>count_words</th>\n",
       "      <th>tw_hash</th>\n",
       "      <th>tw_freq_hash</th>\n",
       "      <th>tw_first_word</th>\n",
       "      <th>tw_second_word</th>\n",
       "      <th>tw_last_word</th>\n",
       "      <th>tw_llast_word</th>\n",
       "      <th>tw_len</th>\n",
       "      <th>tw_hash0</th>\n",
       "      <th>tw_hash1</th>\n",
       "      <th>tw_rt_uhash</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>146255999</th>\n",
       "      <td>0</td>\n",
       "      <td>73443134</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>59</td>\n",
       "      <td>1582081341</td>\n",
       "      <td>1362744</td>\n",
       "      <td>5390</td>\n",
       "      <td>833</td>\n",
       "      <td>0</td>\n",
       "      <td>1546658592</td>\n",
       "      <td>30848700</td>\n",
       "      <td>56</td>\n",
       "      <td>267</td>\n",
       "      <td>0</td>\n",
       "      <td>1428007228</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>146255999</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>76</td>\n",
       "      <td>15</td>\n",
       "      <td>39170347</td>\n",
       "      <td>34858164</td>\n",
       "      <td>28651</td>\n",
       "      <td>210319</td>\n",
       "      <td>2793</td>\n",
       "      <td>27536</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146256000</th>\n",
       "      <td>0</td>\n",
       "      <td>73443135</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>1582091134</td>\n",
       "      <td>179475</td>\n",
       "      <td>17747</td>\n",
       "      <td>1467</td>\n",
       "      <td>0</td>\n",
       "      <td>1172558525</td>\n",
       "      <td>4950585</td>\n",
       "      <td>1837</td>\n",
       "      <td>129</td>\n",
       "      <td>0</td>\n",
       "      <td>1272381192</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>146256000</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>159</td>\n",
       "      <td>29</td>\n",
       "      <td>38847520</td>\n",
       "      <td>1535005</td>\n",
       "      <td>300662</td>\n",
       "      <td>267542</td>\n",
       "      <td>11</td>\n",
       "      <td>867</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146256001</th>\n",
       "      <td>0</td>\n",
       "      <td>73443136</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1582086464</td>\n",
       "      <td>366281</td>\n",
       "      <td>4386</td>\n",
       "      <td>80</td>\n",
       "      <td>0</td>\n",
       "      <td>1457176980</td>\n",
       "      <td>30312854</td>\n",
       "      <td>8</td>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>1235182992</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>146256001</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>72</td>\n",
       "      <td>10</td>\n",
       "      <td>49748666</td>\n",
       "      <td>44369960</td>\n",
       "      <td>7361311</td>\n",
       "      <td>466019</td>\n",
       "      <td>19261</td>\n",
       "      <td>202536</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>58462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146256002</th>\n",
       "      <td>845560</td>\n",
       "      <td>73443137</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1581665518</td>\n",
       "      <td>1030299</td>\n",
       "      <td>4236</td>\n",
       "      <td>4119</td>\n",
       "      <td>0</td>\n",
       "      <td>1524226898</td>\n",
       "      <td>3141261</td>\n",
       "      <td>717</td>\n",
       "      <td>464</td>\n",
       "      <td>0</td>\n",
       "      <td>1501554925</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>146256002</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>161</td>\n",
       "      <td>42</td>\n",
       "      <td>50323632</td>\n",
       "      <td>44880036</td>\n",
       "      <td>7551233</td>\n",
       "      <td>985413</td>\n",
       "      <td>112</td>\n",
       "      <td>2712</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146256003</th>\n",
       "      <td>6845</td>\n",
       "      <td>73443138</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1581799075</td>\n",
       "      <td>6937005</td>\n",
       "      <td>4354</td>\n",
       "      <td>3629</td>\n",
       "      <td>0</td>\n",
       "      <td>1269050894</td>\n",
       "      <td>3141261</td>\n",
       "      <td>717</td>\n",
       "      <td>464</td>\n",
       "      <td>0</td>\n",
       "      <td>1501554925</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>146256003</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>216</td>\n",
       "      <td>71</td>\n",
       "      <td>50323633</td>\n",
       "      <td>44880037</td>\n",
       "      <td>2674107</td>\n",
       "      <td>2769099</td>\n",
       "      <td>646</td>\n",
       "      <td>2378</td>\n",
       "      <td>26</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           hashtags  tweet_id  media  links  domains  tweet_type  language  \\\n",
       "146255999         0  73443134      0      0        0           1        59   \n",
       "146256000         0  73443135      0      0        0           1        54   \n",
       "146256001         0  73443136      9      0        0           1         4   \n",
       "146256002    845560  73443137      0      0        0           2        11   \n",
       "146256003      6845  73443138      7      0        0           2        11   \n",
       "\n",
       "            timestamp  a_user_id  a_follower_count  a_following_count  \\\n",
       "146255999  1582081341    1362744              5390                833   \n",
       "146256000  1582091134     179475             17747               1467   \n",
       "146256001  1582086464     366281              4386                 80   \n",
       "146256002  1581665518    1030299              4236               4119   \n",
       "146256003  1581799075    6937005              4354               3629   \n",
       "\n",
       "           a_is_verified  a_account_creation  b_user_id  b_follower_count  \\\n",
       "146255999              0          1546658592   30848700                56   \n",
       "146256000              0          1172558525    4950585              1837   \n",
       "146256001              0          1457176980   30312854                 8   \n",
       "146256002              0          1524226898    3141261               717   \n",
       "146256003              0          1269050894    3141261               717   \n",
       "\n",
       "           b_following_count  b_is_verified  b_account_creation  b_follows_a  \\\n",
       "146255999                267              0          1428007228            0   \n",
       "146256000                129              0          1272381192            0   \n",
       "146256001                 57              0          1235182992            0   \n",
       "146256002                464              0          1501554925            1   \n",
       "146256003                464              0          1501554925            1   \n",
       "\n",
       "           reply  retweet  retweet_comment  like         id  tr  dt_day  \\\n",
       "146255999      0        0                0     0  146255999   2      19   \n",
       "146256000      0        0                0     0  146256000   2      19   \n",
       "146256001      0        0                0     0  146256001   2      19   \n",
       "146256002      0        0                0     0  146256002   2      14   \n",
       "146256003      0        0                0     0  146256003   2      15   \n",
       "\n",
       "           dt_dow  dt_hour  a_count_combined  a_user_fer_count_delta_time  \\\n",
       "146255999       2        3                 2                            1   \n",
       "146256000       2        5                 0                            0   \n",
       "146256001       2        4                 2                            1   \n",
       "146256002       4        7                 2                            1   \n",
       "146256003       5       20                 2                            1   \n",
       "\n",
       "           a_user_fing_count_delta_time  a_user_fering_count_delta_time  \\\n",
       "146255999                             1                               1   \n",
       "146256000                             1                               0   \n",
       "146256001                             1                               1   \n",
       "146256002                             1                               1   \n",
       "146256003                             1                               1   \n",
       "\n",
       "           a_user_fing_count_mode  a_user_fer_count_mode  \\\n",
       "146255999                       1                      1   \n",
       "146256000                       1                      0   \n",
       "146256001                       1                      1   \n",
       "146256002                       1                      1   \n",
       "146256003                       1                      1   \n",
       "\n",
       "           a_user_fering_count_mode  count_ats  count_char  count_words  \\\n",
       "146255999                         1          0          76           15   \n",
       "146256000                         0          0         159           29   \n",
       "146256001                         1          0          72           10   \n",
       "146256002                         1          0         161           42   \n",
       "146256003                         1          0         216           71   \n",
       "\n",
       "            tw_hash  tw_freq_hash  tw_first_word  tw_second_word  \\\n",
       "146255999  39170347      34858164          28651          210319   \n",
       "146256000  38847520       1535005         300662          267542   \n",
       "146256001  49748666      44369960        7361311          466019   \n",
       "146256002  50323632      44880036        7551233          985413   \n",
       "146256003  50323633      44880037        2674107         2769099   \n",
       "\n",
       "           tw_last_word  tw_llast_word  tw_len  tw_hash0  tw_hash1  \\\n",
       "146255999          2793          27536       6         0         0   \n",
       "146256000            11            867      13         0         0   \n",
       "146256001         19261         202536       5         0         0   \n",
       "146256002           112           2712      14         0         0   \n",
       "146256003           646           2378      26         0         0   \n",
       "\n",
       "           tw_rt_uhash  \n",
       "146255999         7739  \n",
       "146256000        10748  \n",
       "146256001        58462  \n",
       "146256002            0  \n",
       "146256003            0  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tr\n",
       "0    121386431\n",
       "1     12434735\n",
       "2     12434838\n",
       "Name: id, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.groupby('tr')['id'].agg('count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def MultiTE_gpu( tra, col, tar, L=1, smooth_method=0  ):\n",
    "    np.random.seed(L)\n",
    "\n",
    "    cols = col+[tar]\n",
    "    gf = cudf.from_pandas(tra[cols])\n",
    "    mn = gf[tar].mean().astype('float32')\n",
    "    \n",
    "    predtrain = np.zeros( tra.shape[0] )\n",
    "    \n",
    "    for fold in [7,8,9,10,11,12]:\n",
    "        px = np.where( tra.dt_day <fold )[0]\n",
    "        py = np.where( tra.dt_day==fold )[0]\n",
    "        mn = gf[tar].iloc[px].mean().astype('float32')\n",
    "        if smooth_method==0:\n",
    "            te = gf.iloc[px].groupby(col)[tar].agg(['mean','count'])\n",
    "            te['smooth']  = (te['mean']*te['count'])\n",
    "            te['smooth'] += (mn*L)\n",
    "            te['smooth'] /= (te['count']+L)\n",
    "            te = te.drop( ['mean','count'] )\n",
    "        elif smooth_method==1:\n",
    "            te = gf.iloc[px].groupby(col)[tar].agg(['sum','count'])\n",
    "            te['smooth'] = (te['sum']+L) / (te['count']+1)\n",
    "            te = te.drop( ['sum','count'] )\n",
    "        gf2 = gf.iloc[py].copy()\n",
    "        gf2 = gf2.set_index( col )\n",
    "        gf2['id'] = cupy.arange( gf2.shape[0] )\n",
    "        gf2 = gf2.join( te, how='left' )\n",
    "        gf2 = gf2.sort_values( 'id' )\n",
    "        del te, gf2['id']\n",
    "        predtrain[py] = gf2.smooth.fillna(-999).to_array()\n",
    "        del gf2\n",
    "\n",
    "    px = np.where( tra.dt_day <13 )[0]\n",
    "    py = np.where( tra.dt_day>=13 )[0]\n",
    "    mn = gf[tar].iloc[px].mean().astype('float32')\n",
    "    if smooth_method==0:\n",
    "        te = gf.iloc[px].groupby(col)[tar].agg(['mean','count'])\n",
    "        te['smooth']  = (te['mean']*te['count'])\n",
    "        te['smooth'] += (mn*L)\n",
    "        te['smooth'] /= (te['count']+L)\n",
    "        te = te.drop( ['mean','count'] )\n",
    "    elif smooth_method==1:\n",
    "        te = gf.iloc[px].groupby(col)[tar].agg(['sum','count'])\n",
    "        te['smooth'] = (te['sum']+L) / (te['count']+1)\n",
    "        te = te.drop( ['sum','count'] )\n",
    "    gf2 = gf.iloc[py].copy()\n",
    "    gf2 = gf2.set_index( col )\n",
    "    gf2['id'] = cupy.arange( gf2.shape[0] )\n",
    "    gf2 = gf2.join( te, how='left' )\n",
    "    gf2 = gf2.sort_values( 'id' )\n",
    "    del te, gf2['id']\n",
    "    predtrain[py] = gf2.smooth.fillna(-999).to_array()            \n",
    "    del gf2\n",
    "\n",
    "    px = np.where( (tra.dt_day>=7)&(tra.dt_day<=11) )[0]\n",
    "    py = np.where( tra.dt_day==6 )[0]\n",
    "    mn = gf[tar].iloc[px].mean().astype('float32')\n",
    "    if smooth_method==0:\n",
    "        te = gf.iloc[px].groupby(col)[tar].agg(['mean','count'])\n",
    "        te['smooth']  = (te['mean']*te['count'])\n",
    "        te['smooth'] += (mn*L)\n",
    "        te['smooth'] /= (te['count']+L)\n",
    "        te = te.drop( ['mean','count'] )\n",
    "    elif smooth_method==1:\n",
    "        te = gf.iloc[px].groupby(col)[tar].agg(['sum','count'])\n",
    "        te['smooth'] = (te['sum']+L) / (te['count']+1)\n",
    "        te = te.drop( ['sum','count'] )\n",
    "    gf2 = gf.iloc[py].copy()\n",
    "    gf2 = gf2.set_index( col )\n",
    "    gf2['id'] = cupy.arange( gf2.shape[0] )\n",
    "    gf2 = gf2.join( te, how='left' )\n",
    "    gf2 = gf2.sort_values( 'id' )\n",
    "    del te, gf2['id']\n",
    "    predtrain[py] = gf2.smooth.fillna(-999).to_array()            \n",
    "    del gf2\n",
    "    \n",
    "    _ = gc.collect()\n",
    "    predtrain[predtrain <= -999 ] = np.nan\n",
    "    return predtrain.astype(np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hashtags</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>links</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>a_account_creation</th>\n",
       "      <th>b_user_id</th>\n",
       "      <th>b_follower_count</th>\n",
       "      <th>b_following_count</th>\n",
       "      <th>b_is_verified</th>\n",
       "      <th>b_account_creation</th>\n",
       "      <th>b_follows_a</th>\n",
       "      <th>reply</th>\n",
       "      <th>retweet</th>\n",
       "      <th>retweet_comment</th>\n",
       "      <th>like</th>\n",
       "      <th>id</th>\n",
       "      <th>tr</th>\n",
       "      <th>dt_day</th>\n",
       "      <th>dt_dow</th>\n",
       "      <th>dt_hour</th>\n",
       "      <th>a_count_combined</th>\n",
       "      <th>a_user_fer_count_delta_time</th>\n",
       "      <th>a_user_fing_count_delta_time</th>\n",
       "      <th>a_user_fering_count_delta_time</th>\n",
       "      <th>a_user_fing_count_mode</th>\n",
       "      <th>a_user_fer_count_mode</th>\n",
       "      <th>a_user_fering_count_mode</th>\n",
       "      <th>count_ats</th>\n",
       "      <th>count_char</th>\n",
       "      <th>count_words</th>\n",
       "      <th>tw_hash</th>\n",
       "      <th>tw_freq_hash</th>\n",
       "      <th>tw_first_word</th>\n",
       "      <th>tw_second_word</th>\n",
       "      <th>tw_last_word</th>\n",
       "      <th>tw_llast_word</th>\n",
       "      <th>tw_len</th>\n",
       "      <th>tw_hash0</th>\n",
       "      <th>tw_hash1</th>\n",
       "      <th>tw_rt_uhash</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1581262691</td>\n",
       "      <td>0</td>\n",
       "      <td>986</td>\n",
       "      <td>1201</td>\n",
       "      <td>0</td>\n",
       "      <td>1274269909</td>\n",
       "      <td>6237935</td>\n",
       "      <td>94</td>\n",
       "      <td>648</td>\n",
       "      <td>0</td>\n",
       "      <td>1478011810</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "      <td>18</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>68</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>1581497241</td>\n",
       "      <td>1</td>\n",
       "      <td>1225</td>\n",
       "      <td>677</td>\n",
       "      <td>0</td>\n",
       "      <td>1255778244</td>\n",
       "      <td>879586</td>\n",
       "      <td>1139</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>1540395738</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>0</td>\n",
       "      <td>182</td>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>661291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1580978528</td>\n",
       "      <td>2</td>\n",
       "      <td>3016</td>\n",
       "      <td>1623</td>\n",
       "      <td>0</td>\n",
       "      <td>1313450503</td>\n",
       "      <td>647103</td>\n",
       "      <td>780</td>\n",
       "      <td>440</td>\n",
       "      <td>0</td>\n",
       "      <td>1432084055</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>105</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>1581321849</td>\n",
       "      <td>3</td>\n",
       "      <td>2121</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>1547717153</td>\n",
       "      <td>13774339</td>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>0</td>\n",
       "      <td>1534313747</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>103</td>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>198539</td>\n",
       "      <td>2048</td>\n",
       "      <td>616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1580956787</td>\n",
       "      <td>4</td>\n",
       "      <td>813505</td>\n",
       "      <td>200</td>\n",
       "      <td>1</td>\n",
       "      <td>1476348838</td>\n",
       "      <td>13774340</td>\n",
       "      <td>171</td>\n",
       "      <td>388</td>\n",
       "      <td>0</td>\n",
       "      <td>1490166885</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>237</td>\n",
       "      <td>63</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   hashtags  tweet_id  media  links  domains  tweet_type  language  \\\n",
       "0         0         0      0      0        0           2        11   \n",
       "1         3         1      0      0        0           1        11   \n",
       "2         0         2      0      1        1           2        11   \n",
       "3         0         3      0      0        0           1        54   \n",
       "4         0         4      5      0        0           2        11   \n",
       "\n",
       "    timestamp  a_user_id  a_follower_count  a_following_count  a_is_verified  \\\n",
       "0  1581262691          0               986               1201              0   \n",
       "1  1581497241          1              1225                677              0   \n",
       "2  1580978528          2              3016               1623              0   \n",
       "3  1581321849          3              2121                 16              0   \n",
       "4  1580956787          4            813505                200              1   \n",
       "\n",
       "   a_account_creation  b_user_id  b_follower_count  b_following_count  \\\n",
       "0          1274269909    6237935                94                648   \n",
       "1          1255778244     879586              1139                 46   \n",
       "2          1313450503     647103               780                440   \n",
       "3          1547717153   13774339                 1                 45   \n",
       "4          1476348838   13774340               171                388   \n",
       "\n",
       "   b_is_verified  b_account_creation  b_follows_a  reply  retweet  \\\n",
       "0              0          1478011810            0      0        0   \n",
       "1              0          1540395738            1      0        1   \n",
       "2              0          1432084055            1      0        0   \n",
       "3              0          1534313747            0      0        0   \n",
       "4              0          1490166885            0      0        0   \n",
       "\n",
       "   retweet_comment  like  id  tr  dt_day  dt_dow  dt_hour  a_count_combined  \\\n",
       "0                0     0   0   0       9       6       15                18   \n",
       "1                0     1   1   0      12       2        8                -9   \n",
       "2                0     1   2   0       6       3        8                 2   \n",
       "3                0     1   3   0      10       0        8                 2   \n",
       "4                0     1   4   0       6       3        2                 2   \n",
       "\n",
       "   a_user_fer_count_delta_time  a_user_fing_count_delta_time  \\\n",
       "0                           -1                             0   \n",
       "1                           -9                            -9   \n",
       "2                            1                             1   \n",
       "3                            1                             1   \n",
       "4                            1                             1   \n",
       "\n",
       "   a_user_fering_count_delta_time  a_user_fing_count_mode  \\\n",
       "0                              -1                       0   \n",
       "1                              -9                      -9   \n",
       "2                               1                       1   \n",
       "3                               1                       1   \n",
       "4                               1                       1   \n",
       "\n",
       "   a_user_fer_count_mode  a_user_fering_count_mode  count_ats  count_char  \\\n",
       "0                      0                         0          0          68   \n",
       "1                     -9                        -9          0         182   \n",
       "2                      1                         1          0         105   \n",
       "3                      1                         1          2         103   \n",
       "4                      1                         1          0         237   \n",
       "\n",
       "   count_words  tw_hash  tw_freq_hash  tw_first_word  tw_second_word  \\\n",
       "0           17        0             0              0               0   \n",
       "1           37        1             1              1               1   \n",
       "2           24        2             2              2               2   \n",
       "3           22        3             3              3               3   \n",
       "4           63        4             4              4               4   \n",
       "\n",
       "   tw_last_word  tw_llast_word  tw_len  tw_hash0  tw_hash1  tw_rt_uhash  \n",
       "0             0              0       7         0         0            0  \n",
       "1             1              1      12         0         0       661291  \n",
       "2             2              2       6         0         0            0  \n",
       "3             3              3      10    198539      2048          616  \n",
       "4             4              4      16         0         0            0  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TE_b_user_id_tweet_type_language_retweet_comment\n",
      "TE_tw_first_word_tweet_type_language_retweet_comment\n",
      "TE_tw_last_word_tweet_type_language_retweet_comment\n",
      "TE_tw_hash0_tweet_type_language_retweet_comment\n",
      "TE_tw_hash1_tweet_type_language_retweet_comment\n",
      "TE_tw_rt_uhash_tweet_type_language_retweet_comment\n",
      "TE_a_user_id_retweet_comment\n",
      "TE_b_user_id_retweet_comment\n",
      "TE_tw_hash_retweet_comment\n",
      "TE_tw_freq_hash_retweet_comment\n",
      "TE_media_tweet_type_language_a_is_verified_b_is_verified_b_follows_a_retweet_comment\n",
      "TE_a_count_combined_tweet_type_language_retweet_comment\n",
      "TE_a_user_fer_count_delta_time_media_language_retweet_comment\n",
      "TE_a_user_fing_count_delta_time_media_language_retweet_comment\n",
      "TE_a_user_fering_count_delta_time_tweet_type_language_retweet_comment\n",
      "TE_a_user_fing_count_mode_media_language_retweet_comment\n",
      "TE_a_user_fer_count_mode_media_language_retweet_comment\n",
      "TE_a_user_fering_count_mode_tweet_type_language_retweet_comment\n",
      "TE_domains_media_tweet_type_language_retweet_comment\n",
      "TE_links_media_tweet_type_language_retweet_comment\n",
      "TE_hashtags_media_tweet_type_language_retweet_comment\n",
      "CPU times: user 9min 40s, sys: 1min 53s, total: 11min 33s\n",
      "Wall time: 11min 33s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for t in ['retweet_comment']:\n",
    "    for c in [\n",
    "        ['b_user_id','tweet_type','language'],\n",
    "        ['tw_first_word','tweet_type','language'],\n",
    "        ['tw_last_word','tweet_type','language'],\n",
    "        ['tw_hash0','tweet_type','language'],\n",
    "        ['tw_hash1','tweet_type','language'],\n",
    "        ['tw_rt_uhash','tweet_type','language'],\n",
    "        \n",
    "        ['a_user_id'],\n",
    "        ['b_user_id'],\n",
    "        ['tw_hash'],\n",
    "        ['tw_freq_hash'],\n",
    "        \n",
    "        ['media','tweet_type','language','a_is_verified','b_is_verified','b_follows_a'],\n",
    "        ['a_count_combined','tweet_type','language'],\n",
    "        ['a_user_fer_count_delta_time','media','language'],\n",
    "        ['a_user_fing_count_delta_time','media','language'],\n",
    "        ['a_user_fering_count_delta_time','tweet_type','language'],\n",
    "        ['a_user_fing_count_mode','media','language'],\n",
    "        ['a_user_fer_count_mode','media','language'],\n",
    "        ['a_user_fering_count_mode','tweet_type','language'],\n",
    "        \n",
    "        ['domains','media','tweet_type','language'],\n",
    "        ['links','media','tweet_type','language'],\n",
    "        ['hashtags','media','tweet_type','language'],\n",
    "        ]:\n",
    "        fname = 'TE_'+'_'.join(c)+'_'+t\n",
    "        print( fname )\n",
    "        train[fname] = MultiTE_gpu( train, c, t, 20, 0 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hashtags</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>links</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>a_account_creation</th>\n",
       "      <th>b_user_id</th>\n",
       "      <th>b_follower_count</th>\n",
       "      <th>b_following_count</th>\n",
       "      <th>b_is_verified</th>\n",
       "      <th>b_account_creation</th>\n",
       "      <th>b_follows_a</th>\n",
       "      <th>reply</th>\n",
       "      <th>retweet</th>\n",
       "      <th>retweet_comment</th>\n",
       "      <th>like</th>\n",
       "      <th>id</th>\n",
       "      <th>tr</th>\n",
       "      <th>dt_day</th>\n",
       "      <th>dt_dow</th>\n",
       "      <th>dt_hour</th>\n",
       "      <th>a_count_combined</th>\n",
       "      <th>a_user_fer_count_delta_time</th>\n",
       "      <th>a_user_fing_count_delta_time</th>\n",
       "      <th>a_user_fering_count_delta_time</th>\n",
       "      <th>a_user_fing_count_mode</th>\n",
       "      <th>a_user_fer_count_mode</th>\n",
       "      <th>a_user_fering_count_mode</th>\n",
       "      <th>count_ats</th>\n",
       "      <th>count_char</th>\n",
       "      <th>count_words</th>\n",
       "      <th>tw_hash</th>\n",
       "      <th>tw_freq_hash</th>\n",
       "      <th>tw_first_word</th>\n",
       "      <th>tw_second_word</th>\n",
       "      <th>tw_last_word</th>\n",
       "      <th>tw_llast_word</th>\n",
       "      <th>tw_len</th>\n",
       "      <th>tw_hash0</th>\n",
       "      <th>tw_hash1</th>\n",
       "      <th>tw_rt_uhash</th>\n",
       "      <th>TE_b_user_id_tweet_type_language_retweet_comment</th>\n",
       "      <th>TE_tw_first_word_tweet_type_language_retweet_comment</th>\n",
       "      <th>TE_tw_last_word_tweet_type_language_retweet_comment</th>\n",
       "      <th>TE_tw_hash0_tweet_type_language_retweet_comment</th>\n",
       "      <th>TE_tw_hash1_tweet_type_language_retweet_comment</th>\n",
       "      <th>TE_tw_rt_uhash_tweet_type_language_retweet_comment</th>\n",
       "      <th>TE_a_user_id_retweet_comment</th>\n",
       "      <th>TE_b_user_id_retweet_comment</th>\n",
       "      <th>TE_tw_hash_retweet_comment</th>\n",
       "      <th>TE_tw_freq_hash_retweet_comment</th>\n",
       "      <th>TE_media_tweet_type_language_a_is_verified_b_is_verified_b_follows_a_retweet_comment</th>\n",
       "      <th>TE_a_count_combined_tweet_type_language_retweet_comment</th>\n",
       "      <th>TE_a_user_fer_count_delta_time_media_language_retweet_comment</th>\n",
       "      <th>TE_a_user_fing_count_delta_time_media_language_retweet_comment</th>\n",
       "      <th>TE_a_user_fering_count_delta_time_tweet_type_language_retweet_comment</th>\n",
       "      <th>TE_a_user_fing_count_mode_media_language_retweet_comment</th>\n",
       "      <th>TE_a_user_fer_count_mode_media_language_retweet_comment</th>\n",
       "      <th>TE_a_user_fering_count_mode_tweet_type_language_retweet_comment</th>\n",
       "      <th>TE_domains_media_tweet_type_language_retweet_comment</th>\n",
       "      <th>TE_links_media_tweet_type_language_retweet_comment</th>\n",
       "      <th>TE_hashtags_media_tweet_type_language_retweet_comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1581262691</td>\n",
       "      <td>0</td>\n",
       "      <td>986</td>\n",
       "      <td>1201</td>\n",
       "      <td>0</td>\n",
       "      <td>1274269909</td>\n",
       "      <td>6237935</td>\n",
       "      <td>94</td>\n",
       "      <td>648</td>\n",
       "      <td>0</td>\n",
       "      <td>1478011810</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "      <td>18</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>68</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.006546</td>\n",
       "      <td>0.006546</td>\n",
       "      <td>0.003446</td>\n",
       "      <td>0.003143</td>\n",
       "      <td>0.003169</td>\n",
       "      <td>0.003143</td>\n",
       "      <td>0.005977</td>\n",
       "      <td>0.006546</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.002314</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>0.001292</td>\n",
       "      <td>0.000083</td>\n",
       "      <td>0.001497</td>\n",
       "      <td>0.000257</td>\n",
       "      <td>0.000144</td>\n",
       "      <td>0.000166</td>\n",
       "      <td>0.002081</td>\n",
       "      <td>0.002081</td>\n",
       "      <td>0.002543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>1581497241</td>\n",
       "      <td>1</td>\n",
       "      <td>1225</td>\n",
       "      <td>677</td>\n",
       "      <td>0</td>\n",
       "      <td>1255778244</td>\n",
       "      <td>879586</td>\n",
       "      <td>1139</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>1540395738</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>-9</td>\n",
       "      <td>0</td>\n",
       "      <td>182</td>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>661291</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001400</td>\n",
       "      <td>0.003946</td>\n",
       "      <td>0.003858</td>\n",
       "      <td>0.005200</td>\n",
       "      <td>0.001400</td>\n",
       "      <td>0.040914</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.006618</td>\n",
       "      <td>0.005248</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.004026</td>\n",
       "      <td>0.004026</td>\n",
       "      <td>0.003733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1580978528</td>\n",
       "      <td>2</td>\n",
       "      <td>3016</td>\n",
       "      <td>1623</td>\n",
       "      <td>0</td>\n",
       "      <td>1313450503</td>\n",
       "      <td>647103</td>\n",
       "      <td>780</td>\n",
       "      <td>440</td>\n",
       "      <td>0</td>\n",
       "      <td>1432084055</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>105</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.006148</td>\n",
       "      <td>0.024947</td>\n",
       "      <td>0.008098</td>\n",
       "      <td>0.003428</td>\n",
       "      <td>0.003439</td>\n",
       "      <td>0.003428</td>\n",
       "      <td>0.000824</td>\n",
       "      <td>0.005088</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.002667</td>\n",
       "      <td>0.005562</td>\n",
       "      <td>0.006528</td>\n",
       "      <td>0.004926</td>\n",
       "      <td>0.005593</td>\n",
       "      <td>0.004663</td>\n",
       "      <td>0.006182</td>\n",
       "      <td>0.005412</td>\n",
       "      <td>0.005902</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.002729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>1581321849</td>\n",
       "      <td>3</td>\n",
       "      <td>2121</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>1547717153</td>\n",
       "      <td>13774339</td>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>0</td>\n",
       "      <td>1534313747</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>103</td>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>198539</td>\n",
       "      <td>2048</td>\n",
       "      <td>616</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.005433</td>\n",
       "      <td>0.007758</td>\n",
       "      <td>0.005886</td>\n",
       "      <td>0.004415</td>\n",
       "      <td>0.025164</td>\n",
       "      <td>0.005232</td>\n",
       "      <td>0.006727</td>\n",
       "      <td>0.005886</td>\n",
       "      <td>0.005886</td>\n",
       "      <td>0.005144</td>\n",
       "      <td>0.015498</td>\n",
       "      <td>0.014249</td>\n",
       "      <td>0.011357</td>\n",
       "      <td>0.015358</td>\n",
       "      <td>0.010565</td>\n",
       "      <td>0.013418</td>\n",
       "      <td>0.014009</td>\n",
       "      <td>0.006033</td>\n",
       "      <td>0.006033</td>\n",
       "      <td>0.006164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1580956787</td>\n",
       "      <td>4</td>\n",
       "      <td>813505</td>\n",
       "      <td>200</td>\n",
       "      <td>1</td>\n",
       "      <td>1476348838</td>\n",
       "      <td>13774340</td>\n",
       "      <td>171</td>\n",
       "      <td>388</td>\n",
       "      <td>0</td>\n",
       "      <td>1490166885</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>237</td>\n",
       "      <td>63</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.006707</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.003712</td>\n",
       "      <td>0.003428</td>\n",
       "      <td>0.003439</td>\n",
       "      <td>0.003428</td>\n",
       "      <td>0.000879</td>\n",
       "      <td>0.005675</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.005294</td>\n",
       "      <td>0.005562</td>\n",
       "      <td>0.005306</td>\n",
       "      <td>0.004389</td>\n",
       "      <td>0.005593</td>\n",
       "      <td>0.004256</td>\n",
       "      <td>0.005173</td>\n",
       "      <td>0.005412</td>\n",
       "      <td>0.003220</td>\n",
       "      <td>0.003220</td>\n",
       "      <td>0.003089</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   hashtags  tweet_id  media  links  domains  tweet_type  language  \\\n",
       "0         0         0      0      0        0           2        11   \n",
       "1         3         1      0      0        0           1        11   \n",
       "2         0         2      0      1        1           2        11   \n",
       "3         0         3      0      0        0           1        54   \n",
       "4         0         4      5      0        0           2        11   \n",
       "\n",
       "    timestamp  a_user_id  a_follower_count  a_following_count  a_is_verified  \\\n",
       "0  1581262691          0               986               1201              0   \n",
       "1  1581497241          1              1225                677              0   \n",
       "2  1580978528          2              3016               1623              0   \n",
       "3  1581321849          3              2121                 16              0   \n",
       "4  1580956787          4            813505                200              1   \n",
       "\n",
       "   a_account_creation  b_user_id  b_follower_count  b_following_count  \\\n",
       "0          1274269909    6237935                94                648   \n",
       "1          1255778244     879586              1139                 46   \n",
       "2          1313450503     647103               780                440   \n",
       "3          1547717153   13774339                 1                 45   \n",
       "4          1476348838   13774340               171                388   \n",
       "\n",
       "   b_is_verified  b_account_creation  b_follows_a  reply  retweet  \\\n",
       "0              0          1478011810            0      0        0   \n",
       "1              0          1540395738            1      0        1   \n",
       "2              0          1432084055            1      0        0   \n",
       "3              0          1534313747            0      0        0   \n",
       "4              0          1490166885            0      0        0   \n",
       "\n",
       "   retweet_comment  like  id  tr  dt_day  dt_dow  dt_hour  a_count_combined  \\\n",
       "0                0     0   0   0       9       6       15                18   \n",
       "1                0     1   1   0      12       2        8                -9   \n",
       "2                0     1   2   0       6       3        8                 2   \n",
       "3                0     1   3   0      10       0        8                 2   \n",
       "4                0     1   4   0       6       3        2                 2   \n",
       "\n",
       "   a_user_fer_count_delta_time  a_user_fing_count_delta_time  \\\n",
       "0                           -1                             0   \n",
       "1                           -9                            -9   \n",
       "2                            1                             1   \n",
       "3                            1                             1   \n",
       "4                            1                             1   \n",
       "\n",
       "   a_user_fering_count_delta_time  a_user_fing_count_mode  \\\n",
       "0                              -1                       0   \n",
       "1                              -9                      -9   \n",
       "2                               1                       1   \n",
       "3                               1                       1   \n",
       "4                               1                       1   \n",
       "\n",
       "   a_user_fer_count_mode  a_user_fering_count_mode  count_ats  count_char  \\\n",
       "0                      0                         0          0          68   \n",
       "1                     -9                        -9          0         182   \n",
       "2                      1                         1          0         105   \n",
       "3                      1                         1          2         103   \n",
       "4                      1                         1          0         237   \n",
       "\n",
       "   count_words  tw_hash  tw_freq_hash  tw_first_word  tw_second_word  \\\n",
       "0           17        0             0              0               0   \n",
       "1           37        1             1              1               1   \n",
       "2           24        2             2              2               2   \n",
       "3           22        3             3              3               3   \n",
       "4           63        4             4              4               4   \n",
       "\n",
       "   tw_last_word  tw_llast_word  tw_len  tw_hash0  tw_hash1  tw_rt_uhash  \\\n",
       "0             0              0       7         0         0            0   \n",
       "1             1              1      12         0         0       661291   \n",
       "2             2              2       6         0         0            0   \n",
       "3             3              3      10    198539      2048          616   \n",
       "4             4              4      16         0         0            0   \n",
       "\n",
       "   TE_b_user_id_tweet_type_language_retweet_comment  \\\n",
       "0                                          0.006546   \n",
       "1                                               NaN   \n",
       "2                                          0.006148   \n",
       "3                                               NaN   \n",
       "4                                          0.006707   \n",
       "\n",
       "   TE_tw_first_word_tweet_type_language_retweet_comment  \\\n",
       "0                                           0.006546      \n",
       "1                                                NaN      \n",
       "2                                           0.024947      \n",
       "3                                           0.005433      \n",
       "4                                                NaN      \n",
       "\n",
       "   TE_tw_last_word_tweet_type_language_retweet_comment  \\\n",
       "0                                           0.003446     \n",
       "1                                           0.001400     \n",
       "2                                           0.008098     \n",
       "3                                           0.007758     \n",
       "4                                           0.003712     \n",
       "\n",
       "   TE_tw_hash0_tweet_type_language_retweet_comment  \\\n",
       "0                                         0.003143   \n",
       "1                                         0.003946   \n",
       "2                                         0.003428   \n",
       "3                                         0.005886   \n",
       "4                                         0.003428   \n",
       "\n",
       "   TE_tw_hash1_tweet_type_language_retweet_comment  \\\n",
       "0                                         0.003169   \n",
       "1                                         0.003858   \n",
       "2                                         0.003439   \n",
       "3                                         0.004415   \n",
       "4                                         0.003439   \n",
       "\n",
       "   TE_tw_rt_uhash_tweet_type_language_retweet_comment  \\\n",
       "0                                           0.003143    \n",
       "1                                           0.005200    \n",
       "2                                           0.003428    \n",
       "3                                           0.025164    \n",
       "4                                           0.003428    \n",
       "\n",
       "   TE_a_user_id_retweet_comment  TE_b_user_id_retweet_comment  \\\n",
       "0                      0.005977                      0.006546   \n",
       "1                      0.001400                      0.040914   \n",
       "2                      0.000824                      0.005088   \n",
       "3                      0.005232                      0.006727   \n",
       "4                      0.000879                      0.005675   \n",
       "\n",
       "   TE_tw_hash_retweet_comment  TE_tw_freq_hash_retweet_comment  \\\n",
       "0                         NaN                              NaN   \n",
       "1                         NaN                         0.006618   \n",
       "2                         NaN                              NaN   \n",
       "3                    0.005886                         0.005886   \n",
       "4                         NaN                              NaN   \n",
       "\n",
       "   TE_media_tweet_type_language_a_is_verified_b_is_verified_b_follows_a_retweet_comment  \\\n",
       "0                                           0.002314                                      \n",
       "1                                           0.005248                                      \n",
       "2                                           0.002667                                      \n",
       "3                                           0.005144                                      \n",
       "4                                           0.005294                                      \n",
       "\n",
       "   TE_a_count_combined_tweet_type_language_retweet_comment  \\\n",
       "0                                           0.000008         \n",
       "1                                                NaN         \n",
       "2                                           0.005562         \n",
       "3                                           0.015498         \n",
       "4                                           0.005562         \n",
       "\n",
       "   TE_a_user_fer_count_delta_time_media_language_retweet_comment  \\\n",
       "0                                           0.001292               \n",
       "1                                                NaN               \n",
       "2                                           0.006528               \n",
       "3                                           0.014249               \n",
       "4                                           0.005306               \n",
       "\n",
       "   TE_a_user_fing_count_delta_time_media_language_retweet_comment  \\\n",
       "0                                           0.000083                \n",
       "1                                                NaN                \n",
       "2                                           0.004926                \n",
       "3                                           0.011357                \n",
       "4                                           0.004389                \n",
       "\n",
       "   TE_a_user_fering_count_delta_time_tweet_type_language_retweet_comment  \\\n",
       "0                                           0.001497                       \n",
       "1                                                NaN                       \n",
       "2                                           0.005593                       \n",
       "3                                           0.015358                       \n",
       "4                                           0.005593                       \n",
       "\n",
       "   TE_a_user_fing_count_mode_media_language_retweet_comment  \\\n",
       "0                                           0.000257          \n",
       "1                                                NaN          \n",
       "2                                           0.004663          \n",
       "3                                           0.010565          \n",
       "4                                           0.004256          \n",
       "\n",
       "   TE_a_user_fer_count_mode_media_language_retweet_comment  \\\n",
       "0                                           0.000144         \n",
       "1                                                NaN         \n",
       "2                                           0.006182         \n",
       "3                                           0.013418         \n",
       "4                                           0.005173         \n",
       "\n",
       "   TE_a_user_fering_count_mode_tweet_type_language_retweet_comment  \\\n",
       "0                                           0.000166                 \n",
       "1                                                NaN                 \n",
       "2                                           0.005412                 \n",
       "3                                           0.014009                 \n",
       "4                                           0.005412                 \n",
       "\n",
       "   TE_domains_media_tweet_type_language_retweet_comment  \\\n",
       "0                                           0.002081      \n",
       "1                                           0.004026      \n",
       "2                                           0.005902      \n",
       "3                                           0.006033      \n",
       "4                                           0.003220      \n",
       "\n",
       "   TE_links_media_tweet_type_language_retweet_comment  \\\n",
       "0                                           0.002081    \n",
       "1                                           0.004026    \n",
       "2                                                NaN    \n",
       "3                                           0.006033    \n",
       "4                                           0.003220    \n",
       "\n",
       "   TE_hashtags_media_tweet_type_language_retweet_comment  \n",
       "0                                           0.002543      \n",
       "1                                           0.003733      \n",
       "2                                           0.002729      \n",
       "3                                           0.006164      \n",
       "4                                           0.003089      "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.to_parquet( 'data/train-final-te-retweet_comment-1.parquet' )\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Elapsed Time is 17.991000 minutes\n"
     ]
    }
   ],
   "source": [
    "print('Elapsed Time is %f minutes'%((time.time()-startNB)/60))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
